# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

import time
import random
from jdong.user_agents import user_agent_list
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from scrapy.http import HtmlResponse
from logging import getLogger
from jdong.settings import MAX_PAGE_NUM
from scrapy.exceptions import IgnoreRequest


class RandomUserAgentMiddleware(object):
    def process_request(self, request, spider):
        request.headers['User-Agent'] = random.choice(user_agent_list)
        if not request.headers.get('Referer'):
            request.headers['Referer'] = "https://list.jd.com/list.html"


class SeleniumMiddleware(object):
    def __init__(self, timeout=5):
        self.logger = getLogger(__name__)
        self.timeout = timeout
        chrome_options = Options()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        self.browser = webdriver.Chrome(chrome_options=chrome_options)
        self.browser.set_window_size(2000, 5000)
        self.browser.set_page_load_timeout(10)
        self.wait = WebDriverWait(self.browser, self.timeout)

    def __del__(self):
        self.logger.debug("关闭Chrome浏览器")
        self.browser.close()

    def process_request(self, request, spider):
        """
        用Chrome抓取页面
        :param request: Request对象
        :param spider: Spider对象
        :return: HtmlResponse
        """
        self.logger.debug('Chrome启动！')
        page = request.meta.get('page', 1)
        try:
            self.browser.get(request.url)
            if page > 1 and page < MAX_PAGE_NUM:
                input_label = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#page_jump_num')))
                input_label.clear()
                input_label.send_keys(page)

                time.sleep(1)
                self.browser.execute_script("window.scrollBy(0,5000)")
                submit = self.wait.until(EC.presence_of_element_located(
                    (By.XPATH, "//a[@href='javascript:page_jump();']")))
                submit.click()
            if page > MAX_PAGE_NUM:
                self.browser.close()
                raise IgnoreRequest(request)

            self.wait.until(EC.text_to_be_present_in_element((By.XPATH, "//span[@class='fp-text']/b"), str(page)))
            self.wait.until(EC.presence_of_element_located((By.XPATH, "//li[@class='gl-item']")))
            return HtmlResponse(
                url=request.url, body=self.browser.page_source, request=request, encoding='utf-8', status=200)
        except TimeoutException:
            return HtmlResponse(url=request.url, status=500, request=request)

    @classmethod
    def from_crawler(cls, crawler):
        return cls(timeout=crawler.settings.get('SELENIUM_TIMEOUT'))
